package com.gxljc.bear.crawler.itaogao.cbcomcn;

import com.gxljc.bear.crawler.itaogao.ItaogaoConst;
import com.gxljc.bear.crawler.util.DateUtil;
import org.apache.commons.collections.CollectionUtils;
import org.apache.log4j.Logger;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.function.FlatMapFunction;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import java.io.Serializable;
import java.util.*;

/**
 * 中国经营报 seed 数据爬虫。
 *
 * @author tanghaitao
 * @since 2022-9-8
 */
public class CbcomcnUpdateCrawler implements Serializable {
    public static Logger LOG = Logger.getLogger(CbcomcnUpdateCrawler.class);
    private Boolean proxy = false;
    private CbcomcnSeedCrawler crawler = null;
    private Map<String, String> proper = new HashMap<String, String>();

    public CbcomcnUpdateCrawler(Boolean proxy) {
//        proper.put("Cookie", CookieAccount.getInstance().getCbcomcnCookie());
//        proper.put("Cookie", "__jsluid=2319357cf501b43b1a787466f6fad34e; __jsl_clearance=1474556645.531|0|DkHCWQLDmbBYj01fp9cbywAcONc%3D");
        proper.put("Host", "dianzibao.cb.com.cn");
        this.proxy = proxy;
        crawler = new CbcomcnSeedCrawler(proxy);
    }

    public void crawl() throws Exception {
        List<String> seeds = getSeed();
        if (CollectionUtils.isEmpty(seeds)) {
            LOG.error("page is empty");
            return;
        }
        crawlNoSpark(seeds);
    }

    public void crawlNoSpark(List<String> seeds) {
        for (String url : seeds) {
            try {
                crawler.crawl(url);
//                break;
            } catch (Exception e) {
                e.printStackTrace();  //To change body of catch statement use File | Settings | File Templates.
            }
        }
    }

    //构造URL seed
    public List<String> getSeed() throws Exception {
        List<Date> dateSeed = getDateSeed();
        if (CollectionUtils.isEmpty(dateSeed)) return null;
        List<String> urls = new LinkedList<String>();
        final int MAX_NODE = 50;
        for (Date date : dateSeed) {
            String dateFormat = DateUtil.dateFormat(date, "yyyy-MM/dd");
            for (int i = 1; i < MAX_NODE; i++) {
                String url = String.format(ItaogaoConst.NEWSPAPER_SEED.CBCOMCN.getValue(), dateFormat, i + "");
                urls.add(url);
            }

        }
        return urls;
    }

    //获取从当前到1月1号的列表
    private List<Date> getDateSeed() {
        Calendar now = Calendar.getInstance();
        List<Date> list = new LinkedList<>();
        while (true) {
            int week = now.get(Calendar.DAY_OF_WEEK);
            if (week == 2) {    //周一
                list.add(now.getTime());
                return list;
            }
            now.add(Calendar.DATE, -1);
        }
    }

    private List<Date> getDateSeed2() {
        List<Date> list = new LinkedList<>();
        list.add(DateUtil.parseFormat("2016-09-19"));
        return list;
    }

}
